#encoding=utf-8
'''
Created on 2011-11-20

@author: guanyw

python version:2.6

'''

import re
import time
import urllib
import urllib2
import xml.etree.ElementTree as etree

_url = 'http://ir.hit.edu.cn/demo/ltp_ws/ltp'
_encodingUTF8 = 'utf-8'


#===============================================================================
# UrlOpen
#    request an url
#===============================================================================
def UrlOpen(url, args=None, timeout=None,post_data = None):
    if args:
        data = urllib.urlencode(args)     
        url = url + "?" + data
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1',
               'authorization':'username password'} 
    raise ValueError("Please fill the username and password in the headers, and comment this line then continue")
    req = urllib2.Request(url, data=urllib.urlencode(post_data),headers=headers)
    return urllib2.urlopen(req,timeout=timeout)  

#===============================================================================
# __UrlFactoryCallback
#    Get the specific word URL in baike.baidu.com
#===============================================================================
def __UrlFactoryCallback (sWord):

    da = {'c':'utf8','s':sWord}
    
    return (_url, None,da)

#===============================================================================
# __UrlResponseCallback
#===============================================================================
def __UrlResponseCallback(fileHandle, sWord):
    return fileHandle
#    xml = etree.parse(fileHandle)
#    for wd_tag in xml.getiterator('word'):
#        print wd_tag
#===============================================================================
# __Crawler:
#    __Crawler the liWord from web, the URL is formed by call back function 
#===============================================================================
def __Crawler(src):
    timeToSleepFactor = 1    
    while True:
        try:
            sWord = src            
            (wordURL, data,post_data) = __UrlFactoryCallback(sWord)
            
            if wordURL:
                fileHandle = UrlOpen(wordURL, data, 100,post_data=post_data)
                return __UrlResponseCallback(fileHandle, sWord)
        except:
            import traceback;
            traceback.print_exc()
            time.sleep(0.1 * timeToSleepFactor)
            print("s:" + str(0.05 * timeToSleepFactor))
            timeToSleepFactor *= 2

def tag_pairs_from_ltp_ws(rte_pairs):
    #将rte_pairs.text and rte_pairs.hyp 进行分词，词性标注，命名实体标注
    pasg = ''
    per = 50
    for i,p in enumerate(rte_pairs):
        pasg += p.text+'\n'+p.hyp+'\n'
        if i and (i%per == 0 or i >= len(rte_pairs)-1):
            trs_pasg = __Crawler(pasg)
            trsl = __get_info(trs_pasg.read())
#            trsl = __get_info_xml(trs_pasg)
            try:
                for j in range(per+100):
                        rte_pairs[i-j].hyp=trsl[-(2*j+1)]
                        rte_pairs[i-j].text=trsl[-(2*j+2)]
                        print -(2*j+1),rte_pairs[i-j].hyp
                        print -(2*j+2),rte_pairs[i-j].text
            except:
                pass
            pasg = ''
    return rte_pairs

def __get_info(trs_pasg):
    trsl = []

    _sent_tag =r'<para.*?>(.*?)</para>'
    _meaning_tag = r'<word id="\d+" cont="(.*?)" wsd=".*?" wsdexp=".*?" pos="(.*?)" ne="(.*?)"'

    _meaning_parn = re.compile(_meaning_tag,re.DOTALL)
    _sent_parn = re.compile(_sent_tag,re.DOTALL)
    
    sl = re.findall(_sent_parn,trs_pasg)
    for s in sl:
        tagged_s = ''
        ml = re.findall(_meaning_parn, s)
        for m in ml:
            tagged_s += m[0]+' /'+m[1]+' /'+m[2]+' '
        
#        print tagged_s
        trsl.append(tagged_s)
    return trsl

def __get_info_xml(trs_pasg):
    trsl =[]
    with open('temp.xml','w') as fw:
        print trs_pasg.read()
        fw.write(trs_pasg.read().replace('encoding="gbk"','encoding="utf-8"'))
        fw.close()

    xml = etree.parse('temp.xml')
    for s in  xml.getiterator('sent'):
        tagged_s = ''
        for m in s.getiterator('word'):
            tagged_s += m['cont']+' /'+m['pos']+' /'+m['ne']+' '

        trsl.append(tagged_s)
        
    return trsl


if __name__ == '__main__':
    wd = '阿尔马兹别克·阿坦巴耶夫赢得总统选举。\n'*2
    
    tag_pairs_from_ltp_ws()
  